{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "scratchpad",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sayakpaul/Training-BatchNorm-and-Only-BatchNorm/blob/master/All_Layers_Frozen.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lIYdn1woOS1n",
        "colab": {}
      },
      "source": [
        "# Authenticate yourself to use the TPUs\n",
        "import os\n",
        "\n",
        "IS_COLAB_BACKEND = 'COLAB_GPU' in os.environ  # this is always set on Colab, the value is 0 or 1 depending on GPU presence\n",
        "if IS_COLAB_BACKEND:\n",
        "  from google.colab import auth\n",
        "  # Authenticates the Colab machine and also the TPU using your\n",
        "  # credentials so that they can access your private GCS buckets.\n",
        "  auth.authenticate_user()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jT02YRxfsQIN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2ce85f9f-62df-4477-d4ab-3416eb32c8fc"
      },
      "source": [
        "# TensorFlow Imports\n",
        "import tensorflow as tf\n",
        "print(tf.__version__)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2LUjPO4zseS8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Detect hardware\n",
        "try:\n",
        "  tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection\n",
        "except ValueError:\n",
        "  tpu = None\n",
        "  gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n",
        "    \n",
        "# Select appropriate distribution strategy\n",
        "if tpu:\n",
        "  tf.config.experimental_connect_to_cluster(tpu)\n",
        "  tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "  strategy = tf.distribute.experimental.TPUStrategy(tpu) # Going back and forth between TPU and host is expensive. Better to run 128 batches on the TPU before reporting back.\n",
        "  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])  \n",
        "elif len(gpus) > 1:\n",
        "  strategy = tf.distribute.MirroredStrategy([gpu.name for gpu in gpus])\n",
        "  print('Running on multiple GPUs ', [gpu.name for gpu in gpus])\n",
        "elif len(gpus) == 1:\n",
        "  strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n",
        "  print('Running on single GPU ', gpus[0].name)\n",
        "else:\n",
        "  strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n",
        "  print('Running on CPU')\n",
        "print(\"Number of accelerators: \", strategy.num_replicas_in_sync)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JvrSynThsgVg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install wandb -q\n",
        "import wandb\n",
        "wandb.login()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2kHU76rnteAI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://raw.githubusercontent.com/GoogleCloudPlatform/keras-idiomatic-programmer/master/zoo/resnet/resnet_cifar10.py"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kPTxwomhtqw6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Other imports\n",
        "from tensorflow.keras.layers import *\n",
        "from tensorflow.keras.models import *\n",
        "from wandb.keras import WandbCallback\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "import resnet_cifar10\n",
        "import numpy as np\n",
        "import time\n",
        "\n",
        "# Random seed fixation\n",
        "tf.random.set_seed(666)\n",
        "np.random.seed(666)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "399jYCS2ts93",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_training_model():\n",
        "    # ResNet20\n",
        "    n = 2\n",
        "    depth =  n * 9 + 2\n",
        "    n_blocks = ((depth - 2) // 9) - 1\n",
        "\n",
        "    # The input tensor\n",
        "    inputs = Input(shape=(32, 32, 3))\n",
        "\n",
        "    # The Stem Convolution Group\n",
        "    x = resnet_cifar10.stem(inputs)\n",
        "\n",
        "    # The learner\n",
        "    x = resnet_cifar10.learner(x, n_blocks)\n",
        "\n",
        "    # The Classifier for 10 classes\n",
        "    outputs = resnet_cifar10.classifier(x, 10)\n",
        "\n",
        "    # Instantiate the Model\n",
        "    model = Model(inputs, outputs)\n",
        "\n",
        "    model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"sgd\", metrics=[\"accuracy\"])\n",
        "    \n",
        "    return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VCiAgWu5t8N_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "8a64ec92-3dd6-486e-ce49-f23da3a9beb2"
      },
      "source": [
        "# Load the training set of CIFAR10\n",
        "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.cifar10.load_data()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz\n",
            "170500096/170498071 [==============================] - 2s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3YkRBD2Dt_ER",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "BATCH_SIZE = 128 * strategy.num_replicas_in_sync\n",
        "\n",
        "def normalize(image, label):\n",
        "    image = tf.image.convert_image_dtype(image, tf.float32)\n",
        "    label = tf.cast(label, tf.int32)\n",
        "    return image, label\n",
        "\n",
        "def augment(image,label):\n",
        "    image = tf.image.resize_with_crop_or_pad(image, 40, 40) # Add 8 pixels of padding\n",
        "    image = tf.image.random_crop(image, size=[32, 32, 3]) # Random crop back to 28x28\n",
        "    image = tf.image.random_brightness(image, max_delta=0.5) # Random brightness\n",
        "    image = tf.clip_by_value(image, 0., 1.)\n",
        "\n",
        "    return image, label\n",
        "\n",
        "train_ds = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n",
        "train_ds = (\n",
        "    train_ds\n",
        "    .shuffle(1024)\n",
        "    .map(normalize, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "    .map(augment, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
        ")\n",
        "\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test))\n",
        "test_ds = (\n",
        "    test_ds\n",
        "    .map(normalize, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "    .batch(BATCH_SIZE)\n",
        "    .prefetch(tf.data.experimental.AUTOTUNE)\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pbYFGdWnuHya",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ab0fb022-4ac6-4770-a130-7224da4e230e"
      },
      "source": [
        "# Set all the layers to *non-trainable*\n",
        "with strategy.scope():\n",
        "    model = get_training_model()\n",
        "    for layer in model.layers:\n",
        "        layer.trainable = False\n",
        "\n",
        "    model.compile(loss=\"sparse_categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])\n",
        "\n",
        "len(model.trainable_variables)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ygJlCWGquV0y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3ac88c05-aa7e-4f8b-d5db-b9eafe62d8fd"
      },
      "source": [
        "# Train model with a decay schedule\n",
        "wandb.init(project=\"training-bn-only\", id=\"all-layers-frozen\")\n",
        "\n",
        "start = time.time()\n",
        "h = model.fit(train_ds,\n",
        "         validation_data=test_ds,\n",
        "         epochs=75,\n",
        "         callbacks=[WandbCallback()])\n",
        "end = time.time()\n",
        "wandb.log({\"training_time\": end - start})\n",
        "print(\"Network takes {:.3f} seconds to train\".format(end - start))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Logging results to <a href=\"https://wandb.com\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://app.wandb.ai/sayakpaul/training-bn-only\" target=\"_blank\">https://app.wandb.ai/sayakpaul/training-bn-only</a><br/>\n",
              "                Run page: <a href=\"https://app.wandb.ai/sayakpaul/training-bn-only/runs/all-layers-frozen\" target=\"_blank\">https://app.wandb.ai/sayakpaul/training-bn-only/runs/all-layers-frozen</a><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/75\n",
            "49/49 [==============================] - 7s 152ms/step - accuracy: 0.1000 - loss: 9.2736 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 2/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2854 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 3/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2765 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 4/75\n",
            "49/49 [==============================] - 3s 56ms/step - accuracy: 0.1000 - loss: 9.2775 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 5/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2695 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 6/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2926 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 7/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2763 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 8/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2915 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 9/75\n",
            "49/49 [==============================] - 2s 46ms/step - accuracy: 0.1000 - loss: 9.2774 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 10/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2897 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 11/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2800 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 12/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2716 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 13/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2809 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 14/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2835 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 15/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2888 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 16/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2826 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 17/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2725 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 18/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2731 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 19/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2731 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 20/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2654 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 21/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2745 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 22/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2854 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 23/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2830 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 24/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2768 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 25/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2740 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 26/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2794 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 27/75\n",
            "49/49 [==============================] - 3s 58ms/step - accuracy: 0.1000 - loss: 9.2921 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 28/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2876 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 29/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2712 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 30/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.3011 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 31/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2733 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 32/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2838 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 33/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2839 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 34/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2740 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 35/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2643 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 36/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2892 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 37/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2858 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 38/75\n",
            "49/49 [==============================] - 3s 58ms/step - accuracy: 0.1000 - loss: 9.2823 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 39/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2872 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 40/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2828 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 41/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.0999 - loss: 9.2850 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 42/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2835 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 43/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2992 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 44/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2784 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 45/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2863 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 46/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2624 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 47/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2819 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 48/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2838 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 49/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2811 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 50/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2881 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 51/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2929 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 52/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2902 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 53/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2849 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 54/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2845 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 55/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2890 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 56/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2835 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 57/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2961 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 58/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.3069 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 59/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2804 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 60/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2780 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 61/75\n",
            "49/49 [==============================] - 3s 59ms/step - accuracy: 0.1000 - loss: 9.2676 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 62/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2764 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 63/75\n",
            "49/49 [==============================] - 3s 51ms/step - accuracy: 0.1000 - loss: 9.2610 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 64/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2818 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 65/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2867 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 66/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2848 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 67/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2790 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 68/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2867 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 69/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2858 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 70/75\n",
            "49/49 [==============================] - 2s 50ms/step - accuracy: 0.1000 - loss: 9.2855 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 71/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2777 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 72/75\n",
            "49/49 [==============================] - 2s 49ms/step - accuracy: 0.1000 - loss: 9.2618 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 73/75\n",
            "49/49 [==============================] - 2s 47ms/step - accuracy: 0.1000 - loss: 9.2746 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 74/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2935 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Epoch 75/75\n",
            "49/49 [==============================] - 2s 48ms/step - accuracy: 0.1000 - loss: 9.2823 - val_accuracy: 0.1000 - val_loss: 9.9058\n",
            "Network takes 227.113 seconds to train\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}